﻿using HtmlAgilityPack;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Data.Entity.Validation;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace DrugCategoryCrawler
{
    public class WebCrawler
    {
        const string WORKING_POINT = "./workingpoint.json";
        const string URL_ROOT = "http://drugs.medlive.cn/";
        const string URL_ENTRY = URL_ROOT + "drugref/drugCateIndex.do";
        const string URL_GENERATING_CAPTCHA = URL_ROOT + "captchaAction.do";

        List<Category> _categories = new List<Category>();
        List<Drug> _drugs = new List<Drug>();
        System.Net.WebClient _webclient = new System.Net.WebClient();
        char[] _whitechars = " \r\n\t".ToCharArray();
        ClinicDataEntities _db = new ClinicDataEntities();
        CaptchaCracker _captchaCracker = new CaptchaCracker();
        int _drugCnt = 0;

        public void start()
        {
            WorkingPoint workingPoint = loadWorkingPoint();
            if (workingPoint == null)
            {
                parserLevel1Categories();

                var firstLeaf = _db.dic_drug_category.FirstOrDefault(
                    o => !_db.dic_drug_category.Any(x => x.parent == o.code));
                parserDrugList(firstLeaf.url, firstLeaf.code);
            }
            else
            {
                System.Console.WriteLine(
                    "Start from the Working Point{treecode=" + workingPoint.treecode
                    + ",page=" + workingPoint.page + "}");
                startParserDrugList(workingPoint);
            }
        }

        private void parserLevel1Categories()
        {
            string html = getWebContent(URL_ENTRY);

            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);

            string maxCode = _db.dic_drug_category.Where(o => o.parent == null).Max(o => o.code);

            foreach (HtmlNode td in doc.DocumentNode.SelectNodes(
                "//div[contains(@class, 'table1')]//table[1]//tr//td"))
            {
                HtmlNode a = td.SelectSingleNode("a");

                string href = a.Attributes["href"].Value;
                string code = href.Substring(href.LastIndexOf('=') + 1);

                var firstLevelCategory = new Category()
                {
                    name = a.InnerText,
                    code = code,
                    url = href
                };

                _categories.Add(firstLevelCategory);
                saveCategory(firstLevelCategory);

                if (string.Compare(firstLevelCategory.code, maxCode) >= 0)
                    parserSubCategories(firstLevelCategory);
            }
        }

        private void parserSubCategories(Category firstLevelCategory)
        {
            string html = getWebContent(firstLevelCategory.url);

            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);

            if (doc.DocumentNode.SelectSingleNode(
                "//div[contains(@class, 'table_new')]") == null)
            {
                // No sub-categories
                return;
            }

            Category prevSecondCategory = null;
            foreach (HtmlNode tr in doc.DocumentNode.SelectNodes(
                "//div[contains(@class, 'table_new')]//table//tr"))
            {
                if (tr.Attributes.Contains("class"))
                {
                    string className = tr.Attributes["class"].Value;

                    HtmlNode a = tr.SelectSingleNode("th//a");
                    string href = a.Attributes["href"].Value;
                    string code = href.Substring(href.LastIndexOf('=') + 1);
                    string name = a.InnerText;

                    var secondLevelCategory = new Category()
                    {
                        name = name,
                        code = code,
                        url = href,
                        parent = firstLevelCategory.code
                    };

                    prevSecondCategory = secondLevelCategory;

                    _categories.Add(secondLevelCategory);
                    saveCategory(secondLevelCategory);
                }
                else
                {
                    HtmlNode th = tr.SelectSingleNode("th");
                    if (th != null)
                    {
                        // 有第三层分类
                        HtmlNode a = tr.SelectSingleNode("th//a");

                        string href = a.Attributes["href"].Value;
                        string code = href.Substring(href.LastIndexOf('=') + 1);
                        string name = a.InnerText;

                        var thirdLevelCategory = new Category()
                        {
                            name = name,
                            code = code,
                            url = href,
                            parent = prevSecondCategory.code
                        };

                        _categories.Add(thirdLevelCategory);
                        saveCategory(thirdLevelCategory);
                    }
                }
            }
        }

        private void startParserDrugList(WorkingPoint workingPoint)
        {
            List<dic_drug_category> categories = _db.dic_drug_category.ToList();
            bool bStart = false;

            foreach (var cate in categories)
            {
                bool isLeaf = !_db.dic_drug_category.Any(o => o.parent == cate.code);
                if (isLeaf)
                {
                    if (bStart)
                    {
                        parserDrugList(cate.url, cate.code);
                    }
                    else if (cate.code == workingPoint.treecode)
                    {
                        bStart = true;
                        string url = cate.url;
                        if (!string.IsNullOrEmpty(workingPoint.page))
                        {
                            url = url.Replace("drugCate.do", "drugCateLast.do") + "&page=" + workingPoint.page;
                        }
                        parserDrugList(url, cate.code);
                    }
                }
            }
        }

        private void parserDrugList(string url, string categoryCode)
        {
            System.Console.WriteLine("Parsering " + url);

            saveWorkingPoint(url);

            string html = getWebContent(url);

            while (html.IndexOf("验证码") >= 0)
            {
                System.Console.WriteLine("又来验证码啦！drugCount=" + _drugCnt);
                html = _captchaCracker.process(URL_GENERATING_CAPTCHA, url);
                System.Threading.Thread.Sleep(60000);
            }

            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);

            // box-list
            if (doc.DocumentNode.SelectSingleNode(
                "//div[contains(@class, 'box-list')]") != null)
            {
                HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes(
                    "//div[contains(@class, 'medince-name')]//a");
                if (nodes != null)
                {
                    foreach (HtmlNode a in nodes)
                    {
                        string href = a.Attributes["href"].Value;
                        string absoluteHref = getAbsoluteUrl(href);
                        if (!checkExists(absoluteHref, categoryCode))
                        {
                            parserDrugDetail(absoluteHref, categoryCode);
                            ++_drugCnt;
                        }
                    }
                }
            }
            else
            {
                System.Diagnostics.Debug.Write("Not a drug list");
            }

            HtmlNode nextLink = doc.DocumentNode.SelectSingleNode("//a[contains(@title, '下一页')]");
            if (nextLink != null)
            {
                if (nextLink.Attributes.Contains("href") &&
                    !string.IsNullOrEmpty(nextLink.Attributes["href"].Value))
                {
                    parserDrugList(nextLink.Attributes["href"].Value, categoryCode);
                }
            }
        }

        private bool checkExists(string url, string parent)
        {
            //var drug = _drugs.FirstOrDefault(o => o.url == url);
            var drug = _db.dic_drug.FirstOrDefault(o => o.url == url);
            if (drug != null)
            {
                if ((drug.parents + ",").IndexOf(parent + ",") < 0)
                {
                    drug.parents = drug.parents.Replace(parent + ",", "") + "," + parent;
                    updateDrug(drug);
                }
                return true;
            }
            return false;
        }

        private void parserDrugDetail(string url, string parent)
        {
            string html = getWebContent(url);

            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);

            Drug drug = new Drug() { url = url, parents = parent };
            foreach (HtmlNode div in doc.DocumentNode.SelectNodes(
                "//div[contains(@class, 'info-left')]/div"))
            {
                HtmlNodeCollection guigeArr = div.SelectNodes("div[contains(@class, 'gifs')]/span[contains(@class, 'guige')]");
                if (guigeArr != null)
                {
                    foreach (var gg in guigeArr)
                    {
                        drug.guige += gg.InnerText + ",";
                    }
                }
                // 药品名称
                HtmlNode nameDiv = div.SelectSingleNode("div/a[contains(@name, 'genericNameFormat')]");
                if (nameDiv != null)
                {
                    HtmlNode nameNode = div.SelectSingleNode("div/p/label[contains(text(),'通用名称')]");
                    if (nameNode != null)
                        drug.name = removeWhitespaceChar(nameNode.NextSibling.InnerText);
                    HtmlNode nameEnNode = div.SelectSingleNode("div/p/label[contains(text(),'英文名称')]");
                    if (nameEnNode != null)
                        drug.name_en = removeWhitespaceChar(nameEnNode.NextSibling.InnerText);
                    HtmlNode namePyNode = div.SelectSingleNode("div/p/label[contains(text(),'汉语拼音')]");
                    if (namePyNode != null)
                        drug.name_py = removeWhitespaceChar(namePyNode.NextSibling.InnerText);
                    HtmlNode productNameNode = div.SelectSingleNode("div/p/label[contains(text(),'商品名称')]");
                    if (productNameNode != null)
                        drug.product_name = removeWhitespaceChar(productNameNode.NextSibling.InnerText);
                    if (String.IsNullOrEmpty(drug.name))
                        drug.name = drug.product_name;
                }
                else
                {
                    // format simple 1
                    nameDiv = div.SelectSingleNode("div/a[contains(@name, 'genericName')]");
                    if (nameDiv != null)
                    {
                        HtmlNode nameNode = div.SelectSingleNode("div/p/label[contains(text(),'通用名称')]");
                        drug.name = removeWhitespaceChar(nameNode.NextSibling.InnerText);
                        HtmlNode nameEnNode = div.SelectSingleNode("div/p/label[contains(text(),'英文名称')]");
                        if (nameEnNode != null)
                            drug.name_en = removeWhitespaceChar(nameEnNode.NextSibling.InnerText);
                        HtmlNode namePyNode = div.SelectSingleNode("div/p/label[contains(text(),'汉语拼音')]");
                        if (namePyNode != null)
                            drug.name_py = removeWhitespaceChar(namePyNode.NextSibling.InnerText);
                        HtmlNode productNameNode = div.SelectSingleNode("div/p/label[contains(text(),'商品名称')]");
                        if (productNameNode != null)
                            drug.product_name = removeWhitespaceChar(productNameNode.NextSibling.InnerText);
                    }
                }

                readProperty(div, "ingredients", drug, "ingredient");
                readProperty(div, "list_cate", drug, "category_paths");
                readProperty(div, "characters", drug, "character");
                readProperty(div, "indications", drug, "indication");
                readProperty(div, "specification", drug, "specification");
                readProperty(div, "dosageAndAdministration", drug, "usage_and_dosage");
                readProperty(div, "adverseReactions", drug, "adverse_reaction");
                readProperty(div, "contraindications", drug, "contraindication");
                readProperty(div, "cautions", drug, "precaution");
                readProperty(div, "pregnancyAndNursingMothers", drug, "remark_for_pregnant_and_lactating");
                readProperty(div, "pediatricUse", drug, "remark_for_children");
                readProperty(div, "geriatricUse", drug, "remark_for_elder");
                readProperty(div, "list_interaction", drug, "drug_interaction");
                readProperty(div, "overdosage", drug, "overdosage");
                readProperty(div, "pharmacologicalAndToxicological", drug, "pharmacology_and_toxicology");
                readProperty(div, "pharmacokinetics", drug, "pharmacokinetics");
                readProperty(div, "storage", drug, "storage");
                readProperty(div, "package", drug, "packaging");
                readProperty(div, "usefulLife", drug, "validity");
                readProperty(div, "implementStandard", drug, "implement_standard");
                // 批准文号
                readProperty(div, "approvalNo", drug, "approval_no");
                readProperty(div, "corporationID", drug, "manufacturer");
                readProperty(div, "add1", drug, "pregnancy_grading");
                readProperty(div, "nursing_grading", drug, "lactation_grading");
            }
            drug.code = url.Substring(
                url.LastIndexOf('/') + 1,
                url.LastIndexOf('.') - url.LastIndexOf('/') - 1);
            _drugs.Add(drug);
            saveDrug(drug);
        }

        private void readProperty(HtmlNode div, string nodename, Drug drug, string propertyName)
        {
            HtmlNode nameNode = div.SelectSingleNode("div/a[contains(@name, '" + nodename + "')]");
            if (nameNode != null)
            {
                HtmlNode more = div.SelectSingleNode("div[contains(@class,'more-infomation')]/p");
                if (more != null)
                {
                    var property = typeof(Drug).GetProperty(propertyName);
                    property.SetValue(drug, more.InnerText.Trim(_whitechars), null);
                }
            }
        }

        private string getAbsoluteUrl(string url)
        {
            if (!url.StartsWith("http://"))
            {
                url = URL_ROOT + url;
            }
            return url;
        }

        private string getWebContent(string url)
        {
            if (!url.StartsWith("http://"))
            {
                url = URL_ROOT + url;
            }

            // can NOT handle the redirect response and
            // throw an exception about "Too many redirect"
            //var html = _webclient.DownloadString(url);

            string html = "";
            HttpWebRequest webReq = (HttpWebRequest)HttpWebRequest.Create(url);
            try
            {
                webReq.CookieContainer = new CookieContainer();
                webReq.Method = "GET";
                using (WebResponse response = webReq.GetResponse())
                {
                    using (Stream stream = response.GetResponseStream())
                    {
                        StreamReader reader = new StreamReader(stream);
                        html = reader.ReadToEnd();
                    }
                }
            }
            catch (Exception ex)
            {
                System.Console.WriteLine(ex);
                throw ex;
            }
            System.Threading.Thread.Sleep(100);
            return html;
        }

        private string removeUnicodeChar(string src)
        {
            StringBuilder sb = new StringBuilder();
            foreach (char c in src)
            {
                if (c <= 255)
                {
                    sb.Append(c);
                }
            }
            return sb.ToString();
        }

        private string removeWhitespaceChar(string src)
        {
            StringBuilder sb = new StringBuilder();
            foreach (char c in src)
            {
                if (!_whitechars.Contains(c))
                {
                    sb.Append(c);
                }
            }
            return sb.ToString();
        }

        private void saveCategory(Category cate)
        {
            if (_db.dic_drug_category.Any(o => o.code == cate.code))
                return;

            dic_drug_category dcc = new dic_drug_category();

            dcc.code = cate.code;
            dcc.name = cate.name;
            dcc.url = cate.url;
            dcc.parent = cate.parent;

            _db.dic_drug_category.Add(dcc);

            _db.SaveChanges();
        }

        private void saveDrug(Drug drug)
        {
            dic_drug dd = new dic_drug();

            dd.code = drug.code;
            dd.url = drug.url;
            dd.name = drug.name;
            dd.name_en = drug.name_en;
            dd.name_py = drug.name_py;
            dd.product_name = drug.product_name;
            dd.ingredient = drug.ingredient;
            dd.category_paths = drug.category_paths;
            dd.character = drug.character;
            dd.indication = drug.indication;
            dd.specification = drug.specification;
            dd.usage_and_dosage = drug.usage_and_dosage;
            dd.adverse_reaction = drug.adverse_reaction;
            dd.contraindication = drug.contraindication;
            dd.precaution = drug.precaution;
            dd.remark_for_pregnant_and_lactating = drug.remark_for_pregnant_and_lactating;
            dd.remark_for_children = drug.remark_for_children;
            dd.remark_for_elder = drug.remark_for_elder;
            dd.drug_interaction = drug.drug_interaction;
            dd.overdosage = drug.overdosage;
            dd.pharmacology_and_toxicology = drug.pharmacology_and_toxicology;
            dd.pharmacokinetics = drug.pharmacokinetics;
            dd.storage = drug.storage;
            dd.packaging = drug.packaging;
            dd.validity = drug.validity;
            dd.implement_standard = drug.implement_standard;
            dd.approval_no = drug.approval_no;
            dd.manufacturer = drug.manufacturer;
            dd.pregnancy_grading = drug.pregnancy_grading;
            dd.lactation_grading = drug.lactation_grading;
            dd.guige = drug.guige;
            dd.parents = drug.parents;
            dd.update_time = System.DateTime.Now;

            _db.dic_drug.Add(dd);

            try
            {
                _db.SaveChanges();
            }
            catch (DbEntityValidationException dbEx)
            {
                foreach (var validationErrors in dbEx.EntityValidationErrors)
                {
                    foreach (var validationError in validationErrors.ValidationErrors)
                    {
                        Trace.TraceInformation("Property: {0} Error: {1}",
                                                validationError.PropertyName,
                                                validationError.ErrorMessage);
                    }
                }
                throw dbEx;
            }
        }

        private void updateDrug(dic_drug drug)
        {
            dic_drug dd = _db.dic_drug.FirstOrDefault(o => o.code == drug.code);

            dd.parents = drug.parents;
            dd.update_time = System.DateTime.Now;

            _db.Entry(dd).State = System.Data.Entity.EntityState.Modified;

            try
            {
                _db.SaveChanges();
            }
            catch (DbEntityValidationException dbEx)
            {
                foreach (var validationErrors in dbEx.EntityValidationErrors)
                {
                    foreach (var validationError in validationErrors.ValidationErrors)
                    {
                        Trace.TraceInformation("Property: {0} Error: {1}",
                                                validationError.PropertyName,
                                                validationError.ErrorMessage);
                    }
                }
                throw dbEx;
            }
        }

        private StringDictionary getUrlParameters(string url)
        {
            StringDictionary ret = new StringDictionary();

            string sub = url.Substring(url.IndexOf('?') + 1);
            string[] arr1 = sub.Split("&".ToCharArray());
            foreach (string tmp in arr1)
            {
                string[] kvs = tmp.Split("=".ToCharArray());
                if (kvs.Length > 1)
                {
                    ret.Add(kvs[0], kvs[1]);
                }
            }

            return ret;
        }

        private void saveWorkingPoint(string url)
        {
            var parameters = getUrlParameters(url);
            string treecode = parameters["treecode"];
            string page = parameters.ContainsKey("page") ? parameters["page"] : "";
            WorkingPoint wp = new WorkingPoint()
            {
                treecode = treecode,
                page = page
            };
            string jsonStr = JsonConvert.SerializeObject(wp);
            System.IO.File.WriteAllText(WORKING_POINT, jsonStr);
        }

        private WorkingPoint loadWorkingPoint()
        {
            string jsonStr = "";
            if (!System.IO.File.Exists(WORKING_POINT) ||
                String.IsNullOrEmpty(jsonStr = System.IO.File.ReadAllText(WORKING_POINT)))
            {
                return null;
            }
            WorkingPoint wp = JsonConvert.DeserializeObject<WorkingPoint>(jsonStr);
            return wp;
        }
    }

    public class WorkingPoint
    {
        public string treecode { get; set; }
        public string page { get; set; }
    }

    public class Category
    {
        public string code;
        public string name;
        public string url;
        public string parent;
    }

    public class Drug
    {
        public string code { get; set; }
        public string url { get; set; }
        public string name { get; set; }
        public string name_en { get; set; }
        public string name_py { get; set; }
        public string product_name { get; set; } // 商品名称
        public string ingredient { get; set; } // 成份
        public string category_paths { get; set; } // 所属类别
        public string character { get; set; } // 性状
        public string indication { get; set; } // 适应症
        public string specification { get; set; } // 规格
        public string usage_and_dosage { get; set; } // 用法用量
        public string adverse_reaction { get; set; } // 不良反应
        public string contraindication { get; set; } // 禁忌
        public string precaution { get; set; } // 注意事项
        public string remark_for_pregnant_and_lactating { get; set; } // 孕妇及哺乳期妇女用药
        public string remark_for_children { get; set; } // 儿童用药
        public string remark_for_elder { get; set; } // 老年用药
        public string drug_interaction { get; set; } // 药物相互作用
        public string overdosage { get; set; } // 药物过量
        public string pharmacology_and_toxicology { get; set; } // 药理毒理
        public string pharmacokinetics { get; set; } // 药代动力学
        public string storage { get; set; } // 贮藏
        public string packaging { get; set; } // 包装
        public string validity { get; set; } // 有效期
        public string implement_standard { get; set; } // 执行标准
        public string approval_no { get; set; } // 批准文号
        public string manufacturer { get; set; } // 生产企业
        public string pregnancy_grading { get; set; } // 妊娠分级
        public string lactation_grading { get; set; } // 哺乳期分级
        public string guige { get; set; }
        public string parents { get; set; }
    }
}
